package org.berlin.freq;

import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;

public class JavaFreqTool {

    private final String filename;
    private final Map<String, Integer> freq = new HashMap<String, Integer>();
    
    private static final Set<String> STOPWORDS = new HashSet<String>();
    
    private static final int TERMS_FOR_R = 40;     
    private static final int MAX_TERMS_OUTPUT = 180;
    private static final int MIN_TERMS = 2;
    private int totalWords = 0;
    
    
    private static final String FILE_OF_INTEREST = "./docs/SourcesPage.java";
    
    static {
        STOPWORDS.add("that");            
        STOPWORDS.add("have");        
        STOPWORDS.add("they");            
        STOPWORDS.add("the");
        STOPWORDS.add("what");
        STOPWORDS.add("were");        
        STOPWORDS.add("there");
        STOPWORDS.add("well");
        STOPWORDS.add("very");
        STOPWORDS.add("because");
        STOPWORDS.add("about");
        STOPWORDS.add("want");
        STOPWORDS.add("dont");       
        STOPWORDS.add("going");
        STOPWORDS.add("them");
        STOPWORDS.add("been");
        STOPWORDS.add("know");
        STOPWORDS.add("said");
        STOPWORDS.add("then");
        STOPWORDS.add("like");
        STOPWORDS.add("also");
        STOPWORDS.add("when");
        STOPWORDS.add("should");
        STOPWORDS.add("after");
        STOPWORDS.add("those");
        STOPWORDS.add("just");
        STOPWORDS.add("their");
        STOPWORDS.add("some");
        STOPWORDS.add("both");        
        STOPWORDS.add("every");
        STOPWORDS.add("many");        
        STOPWORDS.add("were");        
        STOPWORDS.add("from");
        STOPWORDS.add("thats");
        STOPWORDS.add("youre");
        STOPWORDS.add("weve");        
        STOPWORDS.add("here");
        STOPWORDS.add("will");
        STOPWORDS.add("which");
        STOPWORDS.add("than");
        STOPWORDS.add("much");
        STOPWORDS.add("more");
        STOPWORDS.add("your");
        STOPWORDS.add("these");
        STOPWORDS.add("each");
        STOPWORDS.add("shes");
        STOPWORDS.add("such");
        STOPWORDS.add("sure");
        STOPWORDS.add("only");
        STOPWORDS.add("would");
        STOPWORDS.add("come");
        STOPWORDS.add("whats");        
        STOPWORDS.add("lets");        
        STOPWORDS.add("really");        
        STOPWORDS.add("through");
        STOPWORDS.add("during");
        STOPWORDS.add("came");
        STOPWORDS.add("theyre");
        STOPWORDS.add("theres");
    }
    
    public JavaFreqTool(final String file) {
        this.filename = file;
    }
        
    protected void parseLine(final String line) {           
        // Also, add to frequency set :
        final String split [] = line.split("\\s");
        for (final String wordx : split) {
            String word = wordx.trim().toLowerCase();
            word = word.replaceAll("[^a-zA-Z\\\\.]+", "");
            word = word.trim();
            if (word.length() <= 3) {
                continue;
            }
            // Add total words minus stop words
            totalWords++;
            if (STOPWORDS.contains(word)) {
                continue;
            }
            final Integer ct = freq.get(word);
            if (ct == null) {
                freq.put(word, 1);
            } else {
                freq.put(word, ct.intValue()+1);
            } // End of the if - else //
        } // End of the for //        
    } // End of the method //
    
    private class SortComparator implements Comparator<Integer> {
        @Override
        public int compare(final Integer arg1, final Integer arg2) {
            return arg2.compareTo(arg1);
        }        
    }
    
    protected void sortFreqPrint() {        
        final Map<Integer, List<String>> freqByCt = new HashMap<Integer, List<String>>();
        for (final String word : this.freq.keySet()) {
            final Integer i = this.freq.get(word); 
            final List<String> l = freqByCt.get(i);
            if (l == null) {
                final List<String> nl = new ArrayList<String>();
                nl.add(word);
                freqByCt.put(i, nl);
            } else {
                l.add(word);
            } // End of the if - else //
        } // End of the for //        
        int i = 1;
        final SortedSet<Integer> keys = new TreeSet<Integer>(new SortComparator());
        keys.addAll(freqByCt.keySet());
        
        // First RUN is to get the top count for use with percentage
        i = 1;              
        for (final Integer key : keys) { 
            if (key <= MIN_TERMS) {
                continue;
            }
            final List<String> value = freqByCt.get(key);                                    
            for (final String s : value) {               
               i++;
            } // End of the for //
        } // End of the for //
        
        // Reset counter
        final double maxTermsForPerc = (double) i;
        i = 1;
        for (final Integer ctkey : keys) { 
            if (ctkey <= MIN_TERMS) {
                continue;
            }
            final List<String> value = freqByCt.get(ctkey);                                    
            for (final String s : value) {
               final double perc = ((double)ctkey / maxTermsForPerc) * 100.0;
               System.out.println("id=" + i + " ct=" + ctkey + "(" + String.format("%.2f", perc) + "%) term=" + s);
               i++;
               if (i > MAX_TERMS_OUTPUT) {
                   // Not concerned with any more terms
                   break;
               }
            } // End of the for //
        } // End of the for //
    } // End of the method //
    
    protected void outFreq() {
        PrintWriter pw = null;
        try {
            final BufferedOutputStream s = new BufferedOutputStream(new FileOutputStream("hist.dat"));
            pw = new PrintWriter(s);            
            pw.println();
            this.sortFreqOut(pw);            
        } catch(final Exception e) {
            e.printStackTrace();
        } finally {
            if (pw != null) {
                try {
                    pw.close();
                } catch(Exception ee) {}
            } // End of the if //
        } // End of the try - catch finally //
    } // End of the method //
    
    protected void sortFreqOut(final PrintWriter pw) {
        final Map<Integer, List<String>> freqByCt = new HashMap<Integer, List<String>>();
        for (final String word : this.freq.keySet()) {
            final Integer i = this.freq.get(word); 
            final List<String> l = freqByCt.get(i);
            if (l == null) {
                final List<String> nl = new ArrayList<String>();
                nl.add(word);
                freqByCt.put(i, nl);
            } else {
                l.add(word);
            } // End of the if - else //
        } // End of the for //        
        int i = 1;
        final SortedSet<Integer> keys = new TreeSet<Integer>(new SortComparator());
        keys.addAll(freqByCt.keySet());
        end:
        for (final Integer key : keys) { 
            if (key <= MIN_TERMS) {
                continue;
            }
            final List<String> value = freqByCt.get(key);            
            for (final String s : value) {
                final String sx = s.length() > 8 ? s.substring(0,8) : s;
               pw.println(key + "\t'" + sx);
               i++;
               if (i > TERMS_FOR_R) {
                   break end;
               }
            } // End of the for //                      
        } // End of the for //                 
    } // End of the method //
    
    public void analyze() {
        BufferedReader reader = null;
        try {
            final FileInputStream fis = new FileInputStream(new File(filename));
            reader = new BufferedReader(new InputStreamReader(fis));
            String line = "";    
            while ((line = reader.readLine()) != null) {               
                this.parseLine(line);                
            } // End of the while //    
            System.out.println("-------------------------------");
            System.out.println("Total Count of most terms : " + this.totalWords);
            System.out.println("Interesting Word Freq Count: " + freq.size());
            System.out.println("-------------------------------");
        } catch(final IOException ioe) {
            ioe.printStackTrace();
        } finally {
            if (reader != null) {
                try {
                    reader.close();
                } catch(final IOException ioe) {                   
                }
            } // End of the if //
        } // End of the try - catch finally //        
    } // End of the method //
    
    public static void main(final String [] args) {                   
        final JavaFreqTool wordfreq = new JavaFreqTool(FILE_OF_INTEREST); 
        wordfreq.analyze();        
        wordfreq.sortFreqPrint();
        wordfreq.outFreq();               
    } // End of the method //
    
} // End of the class //
